package com.saibo.spider.gather;

import java.io.File;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.saibo.spider.commons.SubmitRedisCommons;
import com.saibo.spider.utils.FileUtil;
import com.saibo.spider.utils.HttpRequestUtil;
import com.saibo.spider.utils.HttpResponseUtil;
import com.saibo.spider.utils.MD5Util;
import com.saibo.spider.utils.StringUtil;

public class MouldSinaRealTimeHot {
	Logger log = Logger.getLogger(MouldSinaRealTimeHot.class);
	
	public static void main(String[] args) {
		MouldSinaRealTimeHot msrt = new MouldSinaRealTimeHot();
		String url = "http://s.weibo.com/top/summary?cate=realtimehot";
		CloseableHttpResponse httpResponse = HttpRequestUtil.getRequestMethod(url, msrt.requestHeader());
		String html = HttpResponseUtil.getHtml(httpResponse);
		Document document = Jsoup.parse(html);
		Elements scriptElement = document.select("script");
		List<String> jsonlist = msrt.replaceScript(scriptElement);
		for (String string : jsonlist) {
			Map<String,Object> mso = StringUtil.jsonToMap(string);
			if("pl_top_realtimehot".equals(mso.get("pid").toString())){
				List<Map<String,Object>> resultList = new ArrayList<Map<String,Object>>();
				String htmls = mso.get("html").toString();
				Document rankDocument = Jsoup.parse(htmls);
				Elements rankElements = rankDocument.select("div.hot_ranklist>table>tbody>tr");
				long cdate = System.currentTimeMillis();
				for (Element element : rankElements) {
					Map<String,Object> resultMap = new HashMap<String,Object>();
					Element rankEle = element.select("tr>td.td_01>span>em").first();
					Element titleEle = element.select("tr>td.td_02>div>p>a").first();
					Element scoreEle = element.select("tr>td.td_03>p>span").first();
					int rank = Integer.parseInt(StringUtil.element2text(rankEle));
					String title = StringUtil.element2text(titleEle);
					int score = Integer.parseInt(StringUtil.element2text(scoreEle));
					System.out.println(rank+"\t"+title+"\t"+score);
					Date date = new Date(cdate);
					resultMap.put("rowkey", MD5Util.GET16BITMD5(title));
					resultMap.put("title", title);
					resultMap.put("rank", rank);
					resultMap.put("score", score);
					resultMap.put("crawler_time", date);
					resultList.add(resultMap);
				}
				SubmitRedisCommons.submitRedisData("sina_realtimehot_info", StringUtil.objectToJson(resultList));
			}
		}
	}
	
	public List<String> replaceScript(Elements scriptList){
		List<String> list = new ArrayList<String>();
		for (Element element : scriptList) {
			if(element.toString().startsWith("<script>STK && STK.pageletM && STK.pageletM.view(")){
				String json = element.toString();
				json = json.replaceAll("<script>STK && STK\\.pageletM && STK\\.pageletM\\.view\\(", "");
				json = json.replaceAll("\\)</script>", "");
				list.add(json);
			}
		}
		return list;
	}
	
	public Map<String,String> requestHeader(){
		Map<String,String> requestHeader = new HashMap<String,String>();
		requestHeader.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
		requestHeader.put("Accept-Encoding", "gzip, deflate, sdch");
		requestHeader.put("Accept-Language", "zh-CN,zh;q=0.8");
		requestHeader.put("Cache-Control", "max-age=0");
		requestHeader.put("Connection", "keep-alive");
		requestHeader.put("Cookie",FileUtil.txt2String(new File("C:/sinalogincookie.txt")));
		requestHeader.put("Host", "s.weibo.com");
		requestHeader.put("Upgrade-Insecure-Requests", "1");
		requestHeader.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");
		return requestHeader;
	}
}
